# -*- coding: utf-8 -*-

# Scrapy settings for scrapy_jyeoo project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'scrapy_jyeoo'

SPIDER_MODULES = ['scrapy_jyeoo.spiders']
NEWSPIDER_MODULE = 'scrapy_jyeoo.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'scrapy_jyeoo (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',

#    'Host': 'www.jyeoo.com',
#    'Connection': 'keep-alive',
#    'Cache-Control': 'max-age=0',
#    'Upgrade-Insecure-Requests': '1',
#    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
#    'Accept-Encoding': 'gzip, deflate',
#    'Accept-Language': 'zh-CN,zh;q=0.9',
#    'Cookie': 'gsScrollPos-1547=0; gsScrollPos-487=0; JYERN=0.30867223022179924; remind_check=1; QS_ED=1; QS_TP=1; j_math2_q_q_0=b6174fb4-1c52-4186-a1d0-a79ec6087045~acfe19cd-7319-4d79-8409-54462213d84d~; j_math2_q_dg=0; j_math2_q_fg=0; j_math2_q_so=0; j_math2_q_so2=0; j_math2_q_po=0; j_math2_q_pd=1; j_q_lbs=; j_math2_q_ct=0; gsScrollPos-1529=0; gsScrollPos-3250=0; gsScrollPos-439=0; gsScrollPos-463=629; JYERN=0.31502949450733797; gsScrollPos-488=0; gsScrollPos-500=1067; gsScrollPos-1715=577; jyean=yu8gpCmv8H5EUBE1FuxN42BiOPK-ZnfxKh3EqYdflak_XaPniPdd0UOdk1xvZyc-cp7oOvtRGHINC5ZTxlOG_1O4OS9nn3jB9wLElld4ElN-yLM5rABRcs5tQx13oNCn0; UM_distinctid=1612c890b01acf-0506c9c129fb9-393d5f0e-100200-1612c890b02434; LF_Email=gmgjzx1@xyh.com; jy_user_ed_math=14; jy_user_rg_math=; j_math0_q_f=1; gsScrollPos-402=0; JYERN=0.15784073755025796; gsScrollPos-1531=0; jy=BBB6658E6826318D87E037BB3247F9959D5D7F336CF4A26A59AFCD8CCB36E6B2B1C100D685F2C6DEFEA0B67E6DBAED080B47BD1F1C1A5A164E1FB753A7C01BD3449232B9BB39A61AC73EB7B16101CFB7E6277E564DCA47422AEDB33408B1458285B564D8D57504300AD15D79CB547D7E22450080827EC95DC96883B85D763758AD085E02CC5DDC6FE2095C74CED911AB3CAA564A5F0D687DF4091BB3405B17533C14D02A3D9DBC4F6093E126E5F30B49C6CD22DFA79F44FC972678C98123304EF19A6162E2E2F128CC8A225E05C29ECDFFB2882E4B52D61799BC5D0AD2CCB6809AD0D77C04689FD5F131957D505BBF8126DFCA41A2CAB54BD52F68C053CBBE76AAF5D43DE8E31227BE0C0CC7B93CFD0CC777050AC986ECAA3F5F428AB0C951D32F6E7163A482ED4B9DCC51EF8E8AC685D597A2CD7E1BDC0E8B043C0A17B4D09F8F1F6CDE277191552C76551AC5C03BE0; CNZZDATA2018550=cnzz_eid%3D225138185-1516866566-%26ntime%3D1522820008'
#}

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'scrapy_jyeoo.middlewares.ScrapyJyeooSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'scrapy_jyeoo.middleware.CustomHttpProxyMiddleware': 543,
#    'scrapy_jyeoo.middleware.CustomUserAgentMiddleware': 545,
#}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
#    'scrapy_jyeoo.pipelines.ScrapyJyeooPipeline': 300,
#}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.54 Safari/536.5'
